{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Back_propagation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PacktPublishing/Modern-Computer-Vision-with-PyTorch/blob/master/Chapter01/Back_propagation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:49.845714Z",
          "start_time": "2020-09-24T13:36:49.666592Z"
        },
        "id": "TTWK_bQBC7mf"
      },
      "source": [
        "import numpy as np \n",
        "from copy import deepcopy\n",
        "import matplotlib.pyplot as plt\n",
        "x = np.array([[1,1]])\n",
        "y = np.array([[0]])"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:49.850377Z",
          "start_time": "2020-09-24T13:36:49.846832Z"
        },
        "id": "V8f13jAKC9Rv"
      },
      "source": [
        "from copy import deepcopy\n",
        "import numpy as np\n",
        "def feed_forward(inputs, outputs, weights):     \n",
        "    pre_hidden = np.dot(inputs,weights[0])+ weights[1]\n",
        "    hidden = 1/(1+np.exp(-pre_hidden))\n",
        "    out = np.dot(hidden, weights[2]) + weights[3]\n",
        "    mean_squared_error = np.mean(np.square(out - outputs))\n",
        "    return mean_squared_error"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:49.855575Z",
          "start_time": "2020-09-24T13:36:49.851797Z"
        },
        "id": "8mF3nQbMC_Ja"
      },
      "source": [
        "def update_weights(inputs, outputs, weights, lr):\n",
        "    original_weights = deepcopy(weights)\n",
        "    temp_weights = deepcopy(weights)\n",
        "    updated_weights = deepcopy(weights)\n",
        "    original_loss = feed_forward(inputs, outputs, original_weights)\n",
        "    for i, layer in enumerate(original_weights):\n",
        "        for index, weight in np.ndenumerate(layer):\n",
        "            temp_weights = deepcopy(weights)\n",
        "            temp_weights[i][index] += 0.0001\n",
        "            _loss_plus = feed_forward(inputs, outputs, temp_weights)\n",
        "            grad = (_loss_plus - original_loss)/(0.0001)\n",
        "            updated_weights[i][index] -= grad*lr\n",
        "    return updated_weights, original_loss"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:49.859883Z",
          "start_time": "2020-09-24T13:36:49.856643Z"
        },
        "id": "jgka7ZyOC_Ow",
        "scrolled": false
      },
      "source": [
        "W = [\n",
        "    np.array([[-0.0053, 0.3793],\n",
        "              [-0.5820, -0.5204],\n",
        "              [-0.2723, 0.1896]], dtype=np.float32).T, \n",
        "    np.array([-0.0140, 0.5607, -0.0628], dtype=np.float32), \n",
        "    np.array([[ 0.1528, -0.1745, -0.1135]], dtype=np.float32).T, \n",
        "    np.array([-0.5516], dtype=np.float32)\n",
        "]"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:50.261156Z",
          "start_time": "2020-09-24T13:36:49.860952Z"
        },
        "id": "EFr7P1F0C_L_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "outputId": "2a439af3-e2a1-4605-acf1-d8d2b874b1e3"
      },
      "source": [
        "losses = []\n",
        "for epoch in range(100):\n",
        "    W, loss = update_weights(x,y,W,0.01)\n",
        "    losses.append(loss)\n",
        "plt.plot(losses)\n",
        "plt.title('Loss over increasing number of epochs')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Loss over increasing number of epochs')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8dc7+wKEhASEBMIqgqggEfRitVoXXCq2tbdwrdVbr1ar3X+31y6/Lnpb7fV329pWrdRaa1tFa21L3Ze6W5QgVAREwhoiS9i3bJN8fn+cExxiQgaYZJKZz/PxmEfm7J8zZ/KeM99z5hyZGc4555JXWqILcM4517U86J1zLsl50DvnXJLzoHfOuSTnQe+cc0nOg94555KcB72LK0l7JI1MdB2xkPRNSXcnuo4jJckkjU7QssdKWiRpt6QvJqKGthL5evRUGYkuIJlJWgP8h5k9m+hauouZ9Ul0DbEysx8muoYk8HXgeTObmOhCXMd8j94dFkndtpPQnctKZYf5OpcDS+Jdi4svD/oEkJQt6aeS3gsfP5WUHQ4rlvSopB2Stkl6WVJaOOy/JNWEX5OXS/pIB/MvkHSfpFpJayV9W1JauNwdkiZEjVsiqU7SwLD7wvCr+A5Jr0k6PmrcNWENbwF72wuG6K/Nku6VdLukx8KaX5c0KmrcYyU9E67nJknfDPt/T9LDkn4vaRdwRbhOv5a0IXwN/ltSejj+KEl/l7RV0hZJf5DUP2o57b5u4XJ+Hz4fHtZ+uaR14Xy+FTWPXEm/lbRd0jJJX5e0/iDb2CRdI2lF+FreLkltl9tm2Rlh9wvh+r0WNoX9TdKAcL12SZovaXibRZ4vaVVY962t75lwfp8Na94u6SlJ5W3qvE7SCmBFB+tykaQl4Xq8IGlc2P/vwBnAL8I6j25n2oNttyskvSrpF5J2Snon+j0taYikueH7o0rSVVHD0hU0va0Mt+sCSUOjFn1WB6/9aEkvhsvbIunBjrZhUjEzf3TRA1gDnNVO/xuBecBAoAR4DbgpHHYz8EsgM3x8CBAwFqgGhoTjDQdGdbDc+4C/An3D8d4FrgyH3QP8IGrc64Anw+eTgM3AVCAduDxch+yo9VkEDAVyO1i2AaPD5/cCW4EpBM2EfwDmhMP6AhuArwE5YffUcNj3gCbgYoKdkVzgz8BdQH74ur0BfC4cfzRwNpAdvp4vAT8Nh3X4uoXL+X1UfwN+FS7vBKABGBcOvwV4ESgEyoC3gPUH2fYGPAr0B4YBtcD0tstts+yMsPsFoAoYBRQAS8NteFb4Ot4H/KbNsp4HisJlvUvQZAgwI5zXuHDabwOvtZn2mXDaD2xT4Ghgb/j6ZhI01VQBWVG1/sdBXoeDbbcrgAjwlXDenwJ2AkXh8JeAOwjeHxPD1/DMcNh/AovD7atwew2I4bV/APgWwfsqBzg10TnRLVmU6AKS+UHHQb8SOD+q+1xgTfj8RoKQHt1mmtEEIXwWkHmQZaYDjcD4qH6fA14In58FrIwa9irwmfD5nYQfOFHDlwOnR63PZztZ57ZBf3fUsPOBd8Lns4CFHczje8BLUd2DCEI3N6rfLIK24famv7h13gd73Wg/6Muihr8BzAyfrwLOjRr2H3Qe9KdGdT8E3NB2uW2WHR3034oa/r/AE1HdHwUWtVnW9KjuzwPPhc+fIPyQD7vTgH1AedS0Zx5kPf4v8FCb6WuAD0fV2m7Qd7bdCIL+PUBtXvPLCHYmmoG+UcNuBu6Nel/OOIzX/j5gdvR2ToWHN90kxhBgbVT32rAfwK0Ee0xPh1/FbwAwsyrgywQhsVnSHElD+KBigr2jtvMvDZ8/D+RJmhp+/Z9IsNcFQXvr18Kvuzsk7SD4h4teTvUhruvGqOf7gNaDtUMJPvA6Er2ccoJ12hBV110Ee4hIGhS+HjVhU8/vCV6HQ3ndOqt3SJuaYnkdOppXLDZFPa9rp7vtvKLriX4/lQO3Rb1u2wj2gEs7mLatA96rZtYSjl/a4RTvO+h2C9VYmMBtah8CbDOz3W2GtS63s/dPR6/91wnW/42wOeqzMaxHr+dBnxjvEfwTtBoW9sPMdpvZ18xsJHAR8NXWdkszu9/MTg2nNeBH7cx7C0GzR9v514TzaCbYw5kVPh6N+meqJmjW6R/1yDOzB6LmFa/LnVYDBzsNM3o51QR7hsVRdfUzs2PD4T8Mxz/OzPoBnyb4Zw5mFNvr1pkNBE02rYZ2NGIM9gJ5Ud1HHcG8WkXXs//9RPDafa7NNs01s9eixj/YNj3gvRq2dQ8lfD91orPtBlDa2n7epvb3gCJJfdsMa11uNUHT1iExs41mdpWZDSH4pnuHUuBUTA/6rpcpKSfqkUHQTvhtBQdCi4HvEOyFth4MHR2++XcSfH1tUXC+8pkKDtrWE+zVtbRdWFSQ/0BS3/DA21db5x+6n6A99NLweatfAdeEe/uSlC/pgjb/bPHyKDBY0pcVHCTuK2lqeyOa2QbgaeB/JfVTcGB5lKTTw1H6AnuAnZJKCdpvgf3neXf6usXgIeAbkgrDZVx/GPNotQg4TdIwSQXAN45gXq3+M6xtKPAloPUg4y8J6j4W9h8c/eQhzPch4AJJH5GUSXBMpYHguNJBxbDdINi7/6KkzLCuccDjZlYdLuPm8P/meOBK3n8f3w3cJGlM+F49XtKAzmqS9ElJrR/Y2wk+5A7n/dCreNB3vccJwqX18T3gv4FKggN6i4E3w34AY4BnCYLrH8AdZvY8wYHGWwj22DcS/IN0FBBfINhrXAW8QhDm97QONLPXw+FDCNpwW/tXAlcBvyD4J6giaEeNu/BbxNkE7c0bCc74OOMgk3wGyCI4MLkdeBgYHA77PnAiwQfjY8AjUdMdyut2MDcC64HVBNvnYYLAO2Rm9gxBEL8FLCD40DtSfw3ntYjgNfh1uKw/E3yDmRM2a70NnHcItS4n+Ib0c4LX8KPAR82sMcZZHGy7AbxO8J7fAvwAuMTMtobDZhEcv3iPoHnxu/b+b1J+TPAh9DSwK1zf3BjqOQl4XdIeYC7wJTNbFeO69Fo6sHnMORcLSdcSHKg9vdORXbskXUFwIPfURNeS7HyP3rkYSBosaVrY/DCWoAnjz51N51xP4L84dC42WQRnjIwAdgBzCM7xdq7Hi6npRtJ04DaCc7TvNrNb2gy/huCHN80EbctXm9nS8PS9ZQTnvALMM7Nr4la9c865TnUa9Ap+rvwuwYGz9cB8YJaZLY0ap5+Z7QqfXwR83symh0H/qJlN+MCMnXPOdYtYmm6mAFWtR6YlzSH4WfX+oG8N+VA+R3CudXFxsQ0fPvxwJ3fOuZS0YMGCLWZW0t6wWIK+lAN/Obee4FooB5B0HcH52lnAmVGDRkhaSHAK1LfN7OV2pr0auBpg2LBhVFZWxlCWc865VpLWdjQsbmfdmNntZjYK+C+CCydB8GvCYWY2ieBD4H5J/dqZdraZVZhZRUlJux9IzjnnDlMsQV/DgT+vLuPgP3+eQ3BRKcysofXHD2a2gODaFB+4lKlzzrmuE0vQzwfGSBohKQuYSfCLsv0kjYnqvIDwutbhT/xbrz09kuAXcEn/KzTnnOtJOm2jN7OIpOuBpwhOr7zHzJZIuhGoNLO5wPWSziK4mNZ2guuYA5wG3CipieB6EteY2bauWBHnnHPt63GXQKioqDA/GOucc4dG0gIzq2hvmF8CwTnnkpwHvXPOJbmkCfod+xr52XMreLtmZ6JLcc65HiVpLmqWliZ+8uy7tJgxobQg0eU451yPkTR79P1yMjnmqH5Urtme6FKcc65HSZqgB6goL2Thuu1EmpP+zmDOORez5Ar64YXsbWzmnY27Ox/ZOedSRFIF/eTyQgAWrPXmG+eca5VUQV/aP5ej+uVQ6UHvnHP7JVXQS2Ly8EIq1/hVFpxzrlVSBT3ASeWFbNhZT82OukSX4pxzPULSBX3F8CIA36t3zrlQ0gX9MUf1JS8r3Q/IOudcKOmCPiM9jUnD+vsPp5xzLpR0QQ8wubyIdzbuYnd9U6JLcc65hEvKoK8oL6TFYOG6HYkuxTnnEi4pg37SsP6kyQ/IOuccJGnQ983J5NghBcxb7UHvnHNJGfQAJ48sYlH1DuqbmhNdinPOJVTSBv3UEQNojLR4O71zLuUlbdCfNKIICV5fvTXRpTjnXELFFPSSpktaLqlK0g3tDL9G0mJJiyS9Iml81LBvhNMtl3RuPIs/mILcTMYP7se8VR70zrnU1mnQS0oHbgfOA8YDs6KDPHS/mR1nZhOB/wF+HE47HpgJHAtMB+4I59ctTh45gIXrvJ3eOZfaYtmjnwJUmdkqM2sE5gAzokcws11RnfmAhc9nAHPMrMHMVgNV4fy6xdQRRTREWvhntbfTO+dSVyxBXwpUR3WvD/sdQNJ1klYS7NF/8RCnvVpSpaTK2traWGvv1JT97fR+mqVzLnXF7WCsmd1uZqOA/wK+fYjTzjazCjOrKCkpiVdJ9M/L4pijvJ3eOZfaYgn6GmBoVHdZ2K8jc4CLD3PauDt5ZBFvrttOQ8Tb6Z1zqSmWoJ8PjJE0QlIWwcHVudEjSBoT1XkBsCJ8PheYKSlb0ghgDPDGkZcdu6kjBlDf1MJb63d252Kdc67HyOhsBDOLSLoeeApIB+4xsyWSbgQqzWwucL2ks4AmYDtweTjtEkkPAUuBCHCdmXXrrvXUEcGNSOat3MpJ4U1JnHMulcjMOh+rG1VUVFhlZWVc53nebS/TPzeTB64+Oa7zdc65nkLSAjOraG9Y0v4yNtqpowewYO126hq9nd45l3pSIuinjS6msbmF+X7ZYudcCkqJoJ8yoojMdPFq1ZZEl+Kcc90uJYI+LyuDE4cV8ooHvXMuBaVE0AN8aEwxS97bxba9jYkuxTnnulXKBP200cUAvLbS9+qdc6klZYL+uNIC+uZkeDu9cy7lpEzQZ6SnccrIAd5O75xLOSkT9ACnjimmelsd67buS3QpzjnXbVIq6Fvb6X2v3jmXSlIq6EcW5zO4IIeXV8TvmvfOOdfTpVTQS+K0MSW8smILTc0tiS7HOee6RUoFPcCHx5awuyHCwnV+e0HnXGpIuaCfNqaYjDTxwvLNiS7FOee6RcoFfb+cTE4sL+SF5d5O75xLDSkX9BA03yzdsIvNu+oTXYpzznW51Az6owcC8MK7vlfvnEt+KRn04wb3ZVC/bF705hvnXApIyaCXxOlHl/Dyiloifpqlcy7JpWTQA3x47EB21UdYWO2nWTrnkltMQS9puqTlkqok3dDO8K9KWirpLUnPSSqPGtYsaVH4mBvP4o/EtNHFpPtpls65FNBp0EtKB24HzgPGA7MkjW8z2kKgwsyOBx4G/idqWJ2ZTQwfF8Wp7iNWkJvJ5PJCnlvmQe+cS26x7NFPAarMbJWZNQJzgBnRI5jZ82bWeknIeUBZfMvsGmePG8Q7G3ezfrtfzdI5l7xiCfpSoDqqe33YryNXAk9EdedIqpQ0T9LF7U0g6epwnMra2u47E+as8YMAfK/eOZfU4nowVtKngQrg1qje5WZWAfwb8FNJo9pOZ2azzazCzCpKSkriWdJBjSjOZ1RJPs8s3dRty3TOue4WS9DXAEOjusvCfgeQdBbwLeAiM2to7W9mNeHfVcALwKQjqDfuzho/iHmrtrKrvinRpTjnXJeIJejnA2MkjZCUBcwEDjh7RtIk4C6CkN8c1b9QUnb4vBiYBiyNV/HxcPa4QURazH885ZxLWp0GvZlFgOuBp4BlwENmtkTSjZJaz6K5FegD/LHNaZTjgEpJ/wSeB24xsx4V9JOGFTIgP4tnl3nzjXMuOWXEMpKZPQ483qbfd6Ken9XBdK8Bxx1JgV0tPU2cecxAnlqykabmFjLTU/Y3ZM65JOWpRtBOv6s+wvzV2xJdinPOxZ0HPfChMcVkZaTxjDffOOeSkAc9kJeVwYdGF/P0kk2YWaLLcc65uPKgD02fcBQ1O+pYXLMz0aU451xcedCHzh4/iIw08fjijYkuxTnn4sqDPtQ/L4tTRg3gibc3ePONcy6peNBHOf+4wazduo+lG3YluhTnnIsbD/oo54wfRJrgybe9+cY5lzw86KMM6JPNySMH8Nhib75xziUPD/o2zjtuMKtq97Ji855El+Kcc3HhQd/GuccOQoLHF29IdCnOORcXHvRtDOybw0nlRR70zrmk4UHfjo+eMJh3N+3hnY1+9o1zrvfzoG/H+ccNJj1N/HXRe4kuxTnnjpgHfTsG9Mnm1NHFzF30Hi0tfvaNc65386DvwIyJQ6jZUceb67YnuhTnnDsiHvQdOOfYo8jOSPPmG+dcr+dB34E+2RmcNX4Qjy3eQFNzS6LLcc65w+ZBfxAzThjCtr2NvFK1JdGlOOfcYfOgP4jTx5bQLyeDv3nzjXOuF/OgP4jsjHTOP24wTy7ZyN6GSKLLcc65wxJT0EuaLmm5pCpJN7Qz/KuSlkp6S9Jzksqjhl0uaUX4uDyexXeHT0wuY19js1/R0jnXa3Ua9JLSgduB84DxwCxJ49uMthCoMLPjgYeB/wmnLQK+C0wFpgDflVQYv/K7XkV5IeUD8vjjgupEl+Kcc4cllj36KUCVma0ys0ZgDjAjegQze97M9oWd84Cy8Pm5wDNmts3MtgPPANPjU3r3kMQlJ5Yxb9U2qrft63wC55zrYWIJ+lIgend2fdivI1cCTxzKtJKullQpqbK2tjaGkrrXxyeXIcGf3lyf6FKcc+6QxfVgrKRPAxXArYcynZnNNrMKM6soKSmJZ0lxUdo/l2mjivnTm+v9kgjOuV4nlqCvAYZGdZeF/Q4g6SzgW8BFZtZwKNP2BpdMLqN6Wx2vr96W6FKcc+6QxBL084ExkkZIygJmAnOjR5A0CbiLIOQ3Rw16CjhHUmF4EPacsF+vc+6xR9E3O4OHF3jzjXOud+k06M0sAlxPENDLgIfMbImkGyVdFI52K9AH+KOkRZLmhtNuA24i+LCYD9wY9ut1crPSufCEwTy+eAO765sSXY5zzsVMPe0m2BUVFVZZWZnoMtq1qHoHF9/+KjddPIHLTi7vfALnnOsmkhaYWUV7w/yXsYfghLICxg/ux/2vr6OnfUA651xHPOgPgST+beowlm3YxaLqHYkuxznnYuJBf4gunlRKflY6f3h9XaJLcc65mHjQH6I+2RlcNLGUR996j537/KCsc67n86A/DJdOHUZ9UwuPLPRTLZ1zPZ8H/WGYUFrACWUFflDWOdcreNAfpkunlrNi8x7mreqVPwtwzqUQD/rDdNHEIRTmZfKbV1cnuhTnnDsoD/rDlJOZzqwpw3h22Sa/fLFzrkfzoD8Cnz65HEnc9481iS7FOec65EF/BIb0z2X6hKOYM7/a7ynrnOuxPOiP0GenDWd3fYRHFvbKqy8751KAB/0ROnFYIceVFnDvq6v9piTOuR7Jg/4ISeLfpw1nZe1eXny3590G0TnnPOjj4MLjhzC4IIc7X1yZ6FKcc+4DPOjjICsjjStPHcEbq7fx5rrtiS7HOecO4EEfJzOnDKNfTgZ3+V69c66H8aCPkz7ZGXzmlOE8vXQTK2v3JLoc55zbz4M+jq6YNpzM9DRmv7gq0aU459x+HvRxVNwnm09OLuPPC2vYtKs+0eU45xwQY9BLmi5puaQqSTe0M/w0SW9Kiki6pM2wZkmLwsfceBXeU1192kiazfilt9U753qIToNeUjpwO3AeMB6YJWl8m9HWAVcA97czizozmxg+LjrCenu88gH5XDyxlPtfX8dm36t3zvUAsezRTwGqzGyVmTUCc4AZ0SOY2Rozewto6YIae50vnDmaSItx10veVu+cS7xYgr4UqI7qXh/2i1WOpEpJ8yRd3N4Ikq4Ox6msre39vy4dXpzPjIlD+MPra6nd3ZDocpxzKa47DsaWm1kF8G/ATyWNajuCmc02swozqygpKemGkrreF84cQ2OkhdkveVu9cy6xYgn6GmBoVHdZ2C8mZlYT/l0FvABMOoT6eq0RxfnMmFjK7+atZcse36t3ziVOLEE/HxgjaYSkLGAmENPZM5IKJWWHz4uBacDSwy22t/nCmaNpajZuf74q0aU451JYp0FvZhHgeuApYBnwkJktkXSjpIsAJJ0kaT3wSeAuSUvCyccBlZL+CTwP3GJmKRP0I0v68MnJZfxh3jq/3aBzLmFk1rOuoV5RUWGVlZWJLiNuNuys4/RbX+Cjxw/hf//1hESX45xLUpIWhMdDP8B/GdvFBhfkcsW/DOeRhetZvnF3ostxzqUgD/pu8PkPj6JPdga3PrU80aU451KQB3036J+XxTWnj+LZZZuoXLMt0eU451KMB303+fdpwxnUL5ubHlvm95Z1znUrD/pukpeVwdfPPYZ/Vu/gr/+M+WcIzjl3xDzou9HHJpVyfFkBP3piOfsaI4kuxzmXIjzou1FamvjOhePZuKueX/rNSZxz3cSDvptVDC/iwuMHc9eLK6nZUZfocpxzKcCDPgFuOO8YAH742LIEV+KcSwUe9AlQVpjH9WeM5rHFG3jx3d5/WWbnXM/mQZ8gV58+kpHF+Xznr29T39Sc6HKcc0nMgz5BsjPSueniCazduo87X/Br1jvnuo4HfQJNG13MRScM4c4XVrJ6y95El+OcS1Ie9An27QvGkZ2RxjcfWey/mHXOdQkP+gQb2C+Hb14wjn+s2soD89cluhznXBLyoO8BZp40lGmjB3Dz4+/4ufXOubjzoO8BJHHLx4+nucX45iOL6Wk3g3HO9W4e9D3E0KI8/mv6WF58t5aHF6xPdDnOuSTiQd+DfOaU4UwZXsT3/7bU7zHrnIsbD/oeJC1N++8r+9WHFtHsZ+E45+LAg76HGVqUx40zjmX+mu3c9ZL/kMo5d+RiCnpJ0yUtl1Ql6YZ2hp8m6U1JEUmXtBl2uaQV4ePyeBWezD42qZQLjh/Mj59+l7drdia6HOdcL9dp0EtKB24HzgPGA7MkjW8z2jrgCuD+NtMWAd8FpgJTgO9KKjzyspObJH5w8QSK+2TzhQcWsru+KdElOed6sVj26KcAVWa2yswagTnAjOgRzGyNmb0FtLSZ9lzgGTPbZmbbgWeA6XGoO+n1z8vitpkTWbt1L9/wUy6dc0cglqAvBaqjuteH/WIR07SSrpZUKamyttYv29tq6sgBfO2csTz61gZ+/7r/atY5d3h6xMFYM5ttZhVmVlFSUpLocnqUa08fxelHl3DT35Z6e71z7rDEEvQ1wNCo7rKwXyyOZFpHcMrlTz41kaL8LK79wwK2721MdEnOuV4mlqCfD4yRNEJSFjATmBvj/J8CzpFUGB6EPSfs5w5BUX4Wd376RDbtbOD6B94k0tz2UIhzznWs06A3swhwPUFALwMeMrMlkm6UdBGApJMkrQc+CdwlaUk47TbgJoIPi/nAjWE/d4gmDSvkvz82gVertnLLE+8kuhznXC+SEctIZvY48Hibft+Jej6foFmmvWnvAe45ghpd6F8rhrL0vV3c/cpqxg/px8dPbPcld865A/SIg7Eudt+6YBynjBzADX9azPw1/uXIOdc5D/peJjM9jTs/fSJlRblcdV8lq2r3JLok51wP50HfC/XPy+LeK6aQLvHv985n656GRJfknOvBPOh7qWED8vjV5RVs3FnPVfdVsq8xkuiSnHM9lAd9L3bisEJumzmRRdU7uPb3b9IY8dMunXMf5EHfy02fMJibP34cL75b69ewd861K6bTK13P9qmThrFjXxM3P/EO/XIz+cHFE5CU6LKccz2EB32S+Nzpo9hR18SdL6wkM01876JjPeydc4AHfVL5+rljaYq0cPcrq5HEdz863sPeOedBn0wk8a0LxmHAr19ZjQTfudDD3rlU50GfZCTx7QvGYQb3vLqausZmfvCx40hP87B3LlV50CchSfzfC8eRl5XOL56vYndDhJ/860SyMvwkK+dSkQd9kpLE/zl3LH1zMrj5iXfY2xDhjktPJC/LN7lzqcZ38ZLc504fxc0fP46X3q1l5ux51O72yyU4l2o86FPArCnDuOuyCt7dtJuP3fEqVZv9QmjOpRIP+hRx9vhBPHj1KdQ3NfOJO1/jtaotiS7JOddNPOhTyAlD+/PItdMY2Deby+55g3tfXY2ZXzLBuWTnQZ9ihg3I45HP/wtnjC3he39byg1/WkxDpDnRZTnnupAHfQrqm5PJ7Msq+MKZo3mwsppL7vwH1dv2Jbos51wX8aBPUWlp4mvnjOWuyyazZuteLvjZyzyzdFOiy3LOdYGYgl7SdEnLJVVJuqGd4dmSHgyHvy5peNh/uKQ6SYvCxy/jW747UuceexSPfeFDDC3K46r7Kvn+35ZQ3+RNOc4lk06DXlI6cDtwHjAemCVpfJvRrgS2m9lo4CfAj6KGrTSzieHjmjjV7eJo2IA8/nTtv3D5KeX85tU1XHz7qyzfuDvRZTnn4iSWPfopQJWZrTKzRmAOMKPNODOA34bPHwY+Ir+SVq+Sk5nO92dM4DdXnMSWPQ189Bev8KuXVvmNTJxLArEEfSlQHdW9PuzX7jhmFgF2AgPCYSMkLZT0oqQPHWG9roudccxAnvzyaZx+dAk/eHwZl/zyNf+BlXO9XFcfjN0ADDOzScBXgfsl9Ws7kqSrJVVKqqytre3iklxnivtkM/uyydw2cyKrt+zl/J+9zO3PV/k9aZ3rpWIJ+hpgaFR3Wdiv3XEkZQAFwFYzazCzrQBmtgBYCRzddgFmNtvMKsysoqSk5NDXwsWdJGZMLOWZr5zOWeMGcutTyzn/Zy8zb9XWRJfmnDtEsQT9fGCMpBGSsoCZwNw248wFLg+fXwL83cxMUkl4MBdJI4ExwKr4lO66Q0nfbO64dDK/ueIk6puamTl7Hl95cBEbd9YnujTnXIw6vWatmUUkXQ88BaQD95jZEkk3ApVmNhf4NfA7SVXANoIPA4DTgBslNQEtwDVmtq0rVsR1rTOOGcgzI0/nF8+v4Fcvr+bJtzdy7YdHcfVpI8nJTE90ec65g1BPu9ZJRUWFVVZWJroMdxDV2/bxw8eX8cTbGxlckMNXzj6aT5xY5nexci6BJC0ws4r2hvkvY90hG1qUx52fnswDV53MwL7ZfFgZ1GEAAAuvSURBVP3htzjvtpd4aslGv0iacz2QB707bKeMGsBfrpvGHZeeSKTZ+NzvFnDhz1/haQ9853oUb7pxcRFpbuEvi97j539fwdqt+xg3uB/XfngU5084iox0359wrqsdrOnGg97FVWvg3/FCFatq9zKsKI+rPjSCT0wu8/vVOteFPOhdt2tpMZ5euok7X1zJP6t30C8ng1lThnHZKeWUFeYlujznko4HvUsYM2PB2u385tU1PBm23Z95zEAunVrOaUeX+Jk6zsXJwYLev0u7LiWJiuFFVAwvomZHHfe/vpYH56/n2WXzKe2fyyWTy7hkchlDi3wv37mu4nv0rts1Rlp4eulGHpxfzStVWzCDU0YO4OJJQ5g+YTAFuZmJLtG5XsebblyPVbOjjkcWrOdPb65nzdZ9ZGWkccbYEi44fghnHjOQPtn+pdO5WHjQux7PzHhr/U7+sqiGR9/aQO3uBrIz0jjt6BLOGT+Ij4wbRFF+VqLLdK7H8qB3vUpzS3AA9/HFG3jy7Y1s3FVPmqCivIgzjhnIGceUMHZQX/zeNs69z4Pe9Vpmxts1u3hm6UaeWbaZZRt2ATC4IIdTRxdz6phipo0uprhPdoIrdS6xPOhd0ti4s54X393MC8treW3lVnbWNQFwzFF9OXnkAKaOKOKkEUUe/C7leNC7pNTcYrxds5NXqrYwb9VWKtdsp66pGYARxflUlBcyubyQScMKGT2wj5+z75KaB71LCY2RFhbX7KRyzTbmr9nOgrXb2L4v2OPPz0rnuLICji/rz4TSAo4rLaC8KI80D3+XJDzoXUoyM9Zs3cei6u0sXLeDf67fybINu/bf+zY/K51xg/sxbnA/xh7Vl2OO6svRR/WlX46fx+96H/9lrEtJkhhRnM+I4nw+NqkMgKbmFpZv3M3S93axdMMulry3k78srGF3Q2T/dEf1y2HMoD6MKunDqJJ8RpX0YWRJHwb1y/YzfVyv5EHvUkpmehoTSguYUFqwv5+ZUbOjjuUbd7N8026qNu+havMeHqqsZl9j8/7xcjLTGD4gn/IBeZQPyGdoUR5DC3MZWpRHaf9cv6Wi67E86F3Kk0RZYR5lhXl8ZNyg/f3NjI276llVu5dVW/aydste1mzdS9XmPbywvJaGsAmoVXGfbEr75zC4IJfB/XMYUpDLoIIcBhfkMKhvDgP7ZfuHgUsID3rnOiApCO2CXKaNLj5gWEuLUbungXXb9rF++z5qttdRva2O93bWUVW7h5dW1B7wbaBVQW4mA/tmUxI+ivtkM6BPVvA3P4ui/CwG5GdTmJ9Jn+wMbypyceFB79xhSEsTg/rlMKhfDicNL/rAcDNjV32ETbvq2bCznk276qnd3bD/b+3uBhau28HWPQ3sbecDASAzXRTkZlGYl0lhXhYFeZkU5B746JebQb+cTPrlBh8MfXMy6JsTPPfTSV2rmIJe0nTgNiAduNvMbmkzPBu4D5gMbAU+ZWZrwmHfAK4EmoEvmtlTcaveuR5K0v4wPnpQ34OOW9fYzJY9DWzd28i2vQ1s3dPI9n2NbN/XxI59jWzf28SOukaqt+3j7bomdtY1tfttoa3czHTyszPokx38zc/OID8rnbzWv1kZ5Galk5eZTm5W8MjLSicnI52c1r+ZaeRkpoePNHIy0snOTCM7I90/SHqRToNeUjpwO3A2sB6YL2mumS2NGu1KYLuZjZY0E/gR8ClJ44GZwLHAEOBZSUebWefvUudSRG5WenBg9xCuyd8YaWFXfRO76yPsqmtiV30Te+ojQXd9E3sbmtnT0MSehmb2NkTY2xBhT0OELXsa2bdtH3sbmtnXGKGuqZmm5sM7xTojTWRnpJEVPrIz0oPn6WlkZqSRnR70z0xX+DcYlpEuMtOD7ow0kZEejJOR1joseJ6ZLtLDfhlpIj0tmC5N73enpx34PC3sTtP7w9JauyXS0gjGlVA4Tpo4YBwJ0hT0T09TUjSfxbJHPwWoMrNVAJLmADOA6KCfAXwvfP4w8AsFr84MYI6ZNQCrJVWF8/tHfMp3LjVlZaRR3Cc7Lpd6aGpuoa6pmfrGZvY1NlMfaaauMXjUR5qpb2qhvqmZhkjwt76phcZICw2RoF9DpJnGSNCvsbn1r9EYaaauqZld9UG/puZgeFPEiLS00NRsNDW3EGk2mlpa6GE/6TlA2v7wF+jA7tYPhv1/Cb7RKRxPBB8arR8YaWlBv+jxCWbL+CEF/HzWpLjXH0vQlwLVUd3rgakdjWNmEUk7gQFh/3ltpi1tuwBJVwNXAwwbNizW2p1zcdC6d53oH4o1t4TB32JE9v8N+rWY0dQcfEA0txgtLRBpCfpHmo1Ii9HcYjSb0dwc/G1pCfq3WPBobgkOojeH3S0tRosFy20dp7W7tZ7mFsMIjrm0Pm8xwyzoFz1+6zzM2D+NGfvnG/R/f9pgXu8/x2BYUW6XvLY94mCsmc0GZkPwy9gEl+OcS4Cg+cVPP+0KaTGMUwMMjeouC/u1O46kDKCA4KBsLNM655zrQrEE/XxgjKQRkrIIDq7ObTPOXODy8PklwN8tuIjOXGCmpGxJI4AxwBvxKd0551wsOm26CdvcrweeIji98h4zWyLpRqDSzOYCvwZ+Fx5s3UbwYUA43kMEB24jwHV+xo1zznUvv3qlc84lgYNdvTKWphvnnHO9mAe9c84lOQ9655xLch70zjmX5HrcwVhJtcDaI5hFMbAlTuX0Fqm4zpCa652K6wypud6Hus7lZlbS3oAeF/RHSlJlR0eek1UqrjOk5nqn4jpDaq53PNfZm26ccy7JedA751ySS8agn53oAhIgFdcZUnO9U3GdITXXO27rnHRt9M455w6UjHv0zjnnonjQO+dckkuaoJc0XdJySVWSbkh0PV1F0lBJz0taKmmJpC+F/YskPSNpRfi3MNG1xpukdEkLJT0ado+Q9Hq4zR8ML6OdVCT1l/SwpHckLZN0SrJva0lfCd/bb0t6QFJOMm5rSfdI2izp7ah+7W5bBX4Wrv9bkk48lGUlRdBH3cD8PGA8MCu8MXkyigBfM7PxwMnAdeG63gA8Z2ZjgOfC7mTzJWBZVPePgJ+Y2WhgO8FN6pPNbcCTZnYMcALB+ifttpZUCnwRqDCzCQSXRp9Jcm7re4Hpbfp1tG3PI7ifxxiC267eeSgLSoqgJ+oG5mbWCLTewDzpmNkGM3szfL6b4B+/lGB9fxuO9lvg4sRU2DUklQEXAHeH3QLOJLgZPSTnOhcApxHc7wEzazSzHST5tia4T0ZueLe6PGADSbitzewlgvt3ROto284A7rPAPKC/pMGxLitZgr69G5h/4CbkyUbScGAS8DowyMw2hIM2AoMSVFZX+SnwdaAl7B4A7DCzSNidjNt8BFAL/CZssrpbUj5JvK3NrAb4f8A6goDfCSwg+bd1q4627RFlXLIEfcqR1Af4E/BlM9sVPSy8jWPSnDcr6UJgs5ktSHQt3SwDOBG408wmAXtp00yThNu6kGDvdQQwBMjng80bKSGe2zZZgj6lbkIuKZMg5P9gZo+EvTe1fpUL/25OVH1dYBpwkaQ1BM1yZxK0XfcPv95Dcm7z9cB6M3s97H6YIPiTeVufBaw2s1ozawIeIdj+yb6tW3W0bY8o45Il6GO5gXlSCNumfw0sM7MfRw2KvkH75cBfu7u2rmJm3zCzMjMbTrBt/25mlwLPE9yMHpJsnQHMbCNQLWls2OsjBPdfTtptTdBkc7KkvPC93rrOSb2to3S0becCnwnPvjkZ2BnVxNM5M0uKB3A+8C6wEvhWouvpwvU8leDr3FvAovBxPkGb9XPACuBZoCjRtXbR+n8YeDR8PhJ4A6gC/ghkJ7q+LljfiUBluL3/AhQm+7YGvg+8A7wN/A7ITsZtDTxAcByiieDb25UdbVtABGcWrgQWE5yVFPOy/BIIzjmX5JKl6cY551wHPOidcy7JedA751yS86B3zrkk50HvnHNJzoPeOeeSnAe9c84luf8Plsl7Ai5LB/AAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:50.265635Z",
          "start_time": "2020-09-24T13:36:50.262366Z"
        },
        "id": "Hyul1G_CDFy_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f0ce94c6-9902-4c16-def4-0b6b6f9f436c"
      },
      "source": [
        "W"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([[ 0.01424004, -0.5907864 , -0.27549535],\n",
              "        [ 0.39883757, -0.52918637,  0.18640439]], dtype=float32),\n",
              " array([ 0.00554004,  0.5519136 , -0.06599568], dtype=float32),\n",
              " array([[ 0.3475135 ],\n",
              "        [-0.05529078],\n",
              "        [ 0.03760847]], dtype=float32),\n",
              " array([-0.22443289], dtype=float32)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:50.269686Z",
          "start_time": "2020-09-24T13:36:50.266775Z"
        },
        "id": "zWGav-f7DIhI"
      },
      "source": [
        "pre_hidden = np.dot(x,W[0]) + W[1]\n",
        "hidden = 1/(1+np.exp(-pre_hidden))\n",
        "out = np.dot(hidden, W[2]) + W[3]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "ExecuteTime": {
          "end_time": "2020-09-24T13:36:50.273559Z",
          "start_time": "2020-09-24T13:36:50.271055Z"
        },
        "id": "30Wqe4KADQFE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f81dacdc-4efe-4cdf-ef39-e6ffb19e4ca9"
      },
      "source": [
        "out"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.0174781]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QqapuMCwDQiJ"
      },
      "source": [
        ""
      ],
      "execution_count": 8,
      "outputs": []
    }
  ]
}